#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2016-06-18 10:21:55
# @Author  : lichao (qingligongzi@163.com)
# @Link    : http://www.qingligongzi.com
# @Version : 0.0.2

# 网易云音乐热门歌单TOPN
# 使用selenium动态加载网页
# BeautifulSoup解析HTML
# 使用大顶推过滤TOPN

import http_util
import requests
from bs4 import BeautifulSoup
import urllib
import sys
import re
from time import sleep
# selenium
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException

import heapq

class Playlist():
    """歌单"""

    # 头像url
    img_url = ''
    # 歌单名称
    name = ''
    # 歌单id
    p_id = ''
    # 作者id
    author_id = ''
    # 作者昵称
    author_name = ''
    # 播放次数
    played_num = 0

    def __init__(self):
        pass

    def __str__(self):
        output = ''
        pattern = "{0}:\n{1}\n"
        output += pattern.format("头像url",self.img_url)
        output += pattern.format("歌单名称",self.name)
        output += pattern.format("歌单id",self.p_id)
        output += pattern.format("播放次数",self.played_num)
        output += pattern.format("用户id",self.author_id)
        output += pattern.format("用户昵称",self.author_name)
        return output

class Plist():
    """Playlist 集合类"""
    def __init__(self):
        self._array = []
        self._dict = {}

    def push(self,elem):
        self._array.append(elem)
        self._dict[elem.played_num] = elem

    def replace(self,source,target):
        ''' replace source with target '''

        # 如果source为list，取第一个
        if type(source) == list:
            source = source[0]

        self._array.remove(source)
        self._array.append(target)

        source_value = self._dict[source.played_num]
        if type(source_value) == list:
            source_value.remove(source)
            if len(source_value) == 0:
                self._dict.pop(source.played_num)
        else:
            self._dict.pop(source.played_num)

        if self._dict.has_key(target.played_num):
            value = self._dict.get(target.played_num)
            if type(value) == list:
                value.append(target)
            else:
                new_value = [value,target]
                self._dict[target.played_num] = new_value
        else:
            self._dict[target.played_num] = target

        print self._dict
        print '-'*50

    def find(self,played_num):
        return self._dict.get(played_num)

    def data(self):
        return self._array

    def keys(self):
        return self._dict.keys()

class Query():
    """查询参数类"""

    def __init__(self, order="hot", cat="全部", limit=35, offset=0):
        self.order = order
        # 对URL中文编码
        # self.cat = urllib.quote(cat.decode(sys.stdin.encoding).encode("utf-8"))
        self.cat = cat 
        self.limit = limit
        self.offset = offset

    def to_dict(self):
        return dict([("order",self.order),("cat",self.cat),("limit",self.limit),("offset",self.offset)])

class Music163():
    """网易云音乐爬虫类"""

    # TOPN数，默认为10
    top_n = 10

    def __init__(self,query):
        self.playlist_url = "http://music.163.com/discover/playlist/"
        self.playlist_params = query.to_dict()
        self.headers = http_util.headers
        self.headers[http_util.referer] = "http://music.163.com/"

        self.plist = Plist()
        self.top_data = []

    def load_html(self,url,params={},headers={}):
            # 添加chromedriver的目录到PATH
        chrome_driver_path = "D:/repository/chromedriver_win32/"
        sys.path.append(chrome_driver_path)

        # 指定chromedriver路径加载Chrome
        driver = webdriver.Chrome(chrome_driver_path + "chromedriver")

        # Firefox driver is included in the selenium-server-stanalone.jar available in the downloads
        # Firefox不需要安装
        # driver = webdriver.Firefox()

        # 设置浏览器显示位置
        driver.set_window_position(0, 0)
        # 设置浏览器显示大小
        driver.set_window_size(1024,768)

        try:
            r_url = url+"?%s" % urllib.urlencode(params)
            print("请求网页云音乐url:%s"%r_url)
            driver.implicitly_wait(3)
            driver.get(r_url)
        except Exception, e:
            print e
            return None
        self.process_html(driver)

    def process_html(self,driver):
        g_iframe = driver.find_element_by_id("g_iframe")
        print g_iframe
        driver.switch_to.frame(g_iframe)
        # print element.tag_name
        # print element.parent
        # print element.text

        element = driver.find_element_by_id("m-disc-pl-c")
        # print element
        html = element.get_attribute("innerHTML")

        self.parse_html(html)

        driver.implicitly_wait(2)

        #  触发下一页
        try:
            next_btn = driver.find_element_by_xpath("//div[@class='u-page']/a[@class='zbtn znxt']")
        except NoSuchElementException, e:
            print("已经请求完所有歌单")
            driver.close()
            return

        if next_btn is not None:
            # 对页面进行操作，解决点击下一页按钮时触发到播放面板上
            # selenium.common.exceptions.WebDriverException: 
            # Message: unknown error: Element is not clickable at point (682, 629). Other element would receive the click
            # 第一种方式：通过执行js让滚动条滚到底部
            # 第二种方式：通过ActionChains细粒度去控制页面
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            next_btn.click()

            # 需要切回顶级窗口，不然获取不到iframe
            driver.switch_to_default_content()

            # sleep(3)

            driver.implicitly_wait(10)

            self.process_html(driver)

    def parse_html(self,html):
        soup = BeautifulSoup(html)
        li_items = soup.select("ul#m-pl-container > li")
        print("当前页共%s个歌单" % len(li_items))
        for item in li_items:
            playlist = Playlist()

            ucover_div = item.select("div.u-cover")
            if ucover_div is not None and len(ucover_div) > 0:
                ucover_div_img = ucover_div[0].select("img")
                if ucover_div_img is not None and len(ucover_div_img) > 0:
                    playlist.img_url = ucover_div_img[0].attrs["src"]
                    # print("头像url:\n%s\n" % img_url)

                ucover_div_a = ucover_div[0].a
                if ucover_div_a is not None:
                    playlist.name = ucover_div_a.attrs["title"]
                    # print("歌单名称:\n%s\n" % name)
                    href = ucover_div_a.attrs["href"]
                    pattern = re.compile("id=(\d+)", re.S)
                    result = re.search(pattern, href)
                    if result is not None:
                        playlist.p_id = result.group(1)
                        # print("歌单id:\n%s\n" % p_id)
                ucover_nb = ucover_div[0].select("span.nb")
                if ucover_nb is not None and len(ucover_nb) > 0:
                    nb = ucover_nb[0].string
                    # \u4e07 = 万
                    if nb.find('万') > 0:
                        nb = nb.replace('万','0000')

                    playlist.played_num = int(nb)
                    # print("播放次数:\n%s\n" % played_num)

            # pattern = re.compile(r"/user/home?id=(\d+)")
            # user_a = item.find_all(href=pattern)
            user_a = item.select('p a[href^="/user"]')
            if user_a is not None and len(user_a) > 0:
                _href = user_a[0].attrs["href"]
                _result = re.search(pattern, _href)
                if _result is not None:
                    playlist.author_id = _result.group(1)
                    # print("用户id:\n%s\n" % author_id)
                playlist.author_name = user_a[0].string
                # print("用户昵称:\n%s\n" % author_name)

            # 放入歌单list中
            self.push(playlist)

    def push(self,playlist):
        # print self.top_data
        # print self.plist.keys()
        if len(self.top_data) < self.top_n:
            heapq.heappush(self.top_data,playlist.played_num)
            self.plist.push(playlist)
        else:
            top_n_small = self.top_data[0]
            # print 'top_n_small----%s'%top_n_small
            # print 'new_num----%s'%playlist.played_num
            if playlist.played_num > top_n_small:
                p_small = self.plist.find(top_n_small)
                # print 'p_small:%s'%p_small
                heapq.heapreplace(self.top_data,playlist.played_num)
                self.plist.replace(p_small,playlist)

    def start(self):
        reload(sys)
        sys.setdefaultencoding("utf-8")

        self.load_html(self.playlist_url,params = self.playlist_params,headers=self.headers)
        self.output_result()

    def output_result(self):
        for p in self.plist.data():
            print p
            print "-"*50
        f = open('playlist.txt','w')
        try:
            for p in self.plist.data():
                f.write(str(p))
                f.write("-"*50)
        finally:
            f.close()
        
query = Query()
spider = Music163(query)
spider.start()


